package com.yufei.utils;


import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.log4j.Logger;

import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;

/**
 * Created by IntelliJ IDEA.
 * User: mengke
 * Date: 2010-1-26
 * Time: 16:39:34， 据nutch修改
 */
public class EncodingDetector {
	/**
	 * 匹配HTTP头信息中的字符集的正则
	 */
	public static final Pattern PATTERN_ENCODING = Pattern.compile("charset=(.*?)($)", Pattern.CASE_INSENSITIVE);

	/**
	 * 匹配页面代码中字符集的正则
	 */
	public static final Pattern PATTERN_HEAD_ENCODING = Pattern.compile("&lt;head&gt;.*?&lt;/head&gt;", Pattern.CASE_INSENSITIVE);

	/**
	 * 匹配页面代码中HEAD标签的正则
	 */
	public static final Pattern PATTERN_HEADER = Pattern.compile("&lt;meta.*charset=(.*?)(\"|\').*", Pattern.DOTALL);

    private class EncodingClue {
        private String value;
        private String source;
        private int confidence;

        //忽略字符集可信度
        public EncodingClue(String value, String source) {
            this(value, source, NO_THRESHOLD);
        }

        public EncodingClue(String value, String source, int confidence) {
            this.value = value.toLowerCase();
            this.source = source;
            this.confidence = confidence;
        }

        public String toString() {
            return value + " (" + source +
                    ((confidence >= 0) ? ", " + confidence + "% confidence" : "") + ")";
        }

        public boolean isEmpty() {
            return (value == null || "".equals(value));
        }

        public boolean meetsThreshold() {
            return (confidence < 0 ||
                    (minConfidence >= 0 && confidence >= minConfidence));
        }
    }

    private static final Logger log = Logger.getLogger(EncodingDetector.class);

    public static final int NO_THRESHOLD = -1;

    public static final String MIN_CONFIDENCE_KEY = "block.charset.min.confidence";

    private static final HashMap<String, String> ALIASES = new HashMap<String, String>();

    private static final HashSet<String> DETECTABLES = new HashSet<String>();

    // 页面大小小于该值，程序不做处理
    private static final int MIN_LENGTH = 4;

    static {
        DETECTABLES.add("text/html");
        DETECTABLES.add("text/plain");
        DETECTABLES.add("text/richtext");
        DETECTABLES.add("text/rtf");
        DETECTABLES.add("text/sgml");
        DETECTABLES.add("text/tab-separated-values");
        DETECTABLES.add("text/xml");
        DETECTABLES.add("application/rss+xml");
        DETECTABLES.add("application/xhtml+xml");

        //ALIASES.put("ISO-8859-1", "windows-1252");
        ALIASES.put("EUC-KR", "x-windows-949");
        ALIASES.put("x-EUC-CN", "GB18030");
        //ALIASES.put("GBK", "GB18030");

    }
    //
    private int  minConfidence =50;

    private CharsetDetector detector;

    private List<EncodingClue> clues;

    public EncodingDetector() {
       
        detector = new CharsetDetector();
        clues = new ArrayList<EncodingClue>();
    }

    public void autoDetectClues(byte[] data, String contentType, boolean filter) {

        if (minConfidence >= 0 && DETECTABLES.contains(contentType)
                && data.length > MIN_LENGTH) {
            CharsetMatch[] matches = null;

            //该处使用IBM的国际化开发包
            try {
                detector.enableInputFilter(filter);
                if (data.length > MIN_LENGTH) {
                    detector.setText(data);
                    matches = detector.detectAll();
                }
            } catch (Exception e) {
                log.debug("Exception from ICU4J (ignoring): ", e);
            }

            if (matches != null) {
                for (CharsetMatch match : matches) {
                    //将所有可能的字符集以及可信度存到List中
                    addClue(match.getName(), "detect", match.getConfidence());
                }
            }
        }

        // 将HTTP请求头信息中或页面头信息中的字符编码加入到List中
       addClue(parseCharacterEncoding(data, contentType), "header");
    }

    public void addClue(String value, String source, int confidence) {
        if (value == null || "".equals(value)) {
            return;
        }
        value = resolveEncodingAlias(value);
        if (value != null) {
            clues.add(new EncodingClue(value, source, confidence));
        }
    }

    public void addClue(String value, String source) {
        addClue(value, source, NO_THRESHOLD);
    }

    /**
     * 获取baseUrl的最有可能字符集
     * @param baseUrl 访问URL
     * @param defaultValue 默认字符集
     * @return baseUrl的最有可能字符集
     * 字符集概率大于客户配置 的最小可信度的话就直接返回此字符集，否则使用默认的字符集（也即是从返回头部信息以及网页内容中解析出来的字符集）
     * 
     */
    public String guessEncoding(String baseUrl, String defaultValue) {

        if (log.isTraceEnabled()) {
            findDisagreements(baseUrl, clues);
        }

        EncodingClue defaultClue = new EncodingClue(defaultValue, "default");
        EncodingClue bestClue = defaultClue;

        for (EncodingClue clue : clues) {
            if (log.isTraceEnabled()) {
                log.trace(baseUrl + ": charset " + clue);
            }
            String charset = clue.value;
            if (minConfidence >= 0 && clue.confidence >= minConfidence) {
                //如果当前字符集的可信度大于配置的最小可信度，则对该字符集进行处理
                if (log.isTraceEnabled()) {
                    log.trace(baseUrl + ": Choosing encoding: " + charset +
                            " with confidence " + clue.confidence);
                }
                return resolveEncodingAlias(charset).toLowerCase();
            } else if (clue.confidence == NO_THRESHOLD && bestClue == defaultClue) {
                bestClue = clue;
            }
        }

        if (log.isTraceEnabled()) {
            log.trace(baseUrl + ": Choosing encoding: " + bestClue);
        }
        return bestClue.value.toLowerCase();
    }

    private void findDisagreements(String url, List<EncodingClue> newClues) {
        HashSet<String> valsSeen = new HashSet<String>();
        HashSet<String> sourcesSeen = new HashSet<String>();
        boolean disagreement = false;
        for (EncodingClue clue : newClues) {
            if (!clue.isEmpty() && !sourcesSeen.contains(clue.source)) {
                if (valsSeen.size() > 0 && !valsSeen.contains(clue.value)
                        && clue.meetsThreshold()) {
                    disagreement = true;
                }
                if (clue.meetsThreshold()) {
                    valsSeen.add(clue.value);
                }
                sourcesSeen.add(clue.source);
            }
        }
        if (disagreement) {
            StringBuffer sb = new StringBuffer();
            sb.append("Disagreement: ").append(url).append("; ");
            for (int i = 0; i < newClues.size(); i++) {
                if (i > 0) {
                    sb.append(", ");
                }
                sb.append(newClues.get(i));
            }
            log.trace(sb.toString());
        }
    }

    public static String resolveEncodingAlias(String encoding) {
        if (encoding == null || !Charset.isSupported(encoding))
            return null;
        String canonicalName = Charset.forName(encoding).name();
        return ALIASES.containsKey(canonicalName) ? ALIASES.get(canonicalName)
                : canonicalName;
    }

    /**
     * 根据HTTP请求头信息或者网页元信息中获取字符集
     * @param content 页面内容（字符数组）
     * @param contentType 头信息中的contentType
     * 优先取返回结果中的字符集
     * 如果返回结果中没有接着会到网页内容中去取
     * @return 字符集
     */
    public static String parseCharacterEncoding(byte[] content, String contentType) {
        String encoding = null;
        if (contentType != null && contentType.length() > 0 && !"".equals(contentType)) {
            Matcher match = PATTERN_ENCODING.matcher(contentType);
            if (match.find()) {
                encoding = match.group(1);
            }
        }
        if (encoding == null || encoding.length() == 0 || encoding.equals("")) {
            String htmlContent = new String(content);
            Matcher match =PATTERN_HEADER.matcher(htmlContent);
            if (match.find()) {
                htmlContent = match.group();
                match = PATTERN_HEAD_ENCODING.matcher(htmlContent);
                if (match.find()) {
                    encoding = match.group(1);
                }
            } else {
                match = PATTERN_HEAD_ENCODING.matcher(htmlContent);
                if (match.find()) {
                    encoding = match.group(1);
                }
            }
        }
        if(encoding == null) return null;
        encoding = encoding.trim();
        if ((encoding.length() > 2) && (encoding.startsWith("\""))
                    && (encoding.endsWith("\""))) {
            encoding = encoding.substring(1, encoding.length() - 1);
        }
        if(encoding.indexOf("/") > 0) {
            encoding = encoding.substring(0, encoding.indexOf("/"));
        }
        if(encoding.indexOf(">") > 0) {
            encoding = encoding.substring(0, encoding.indexOf(">"));
        }

        return encoding.trim();
    }

    //测试用
    private static int calculateTryToRead(int totalRead) {
        int tryToRead = 8 * 1024;
        if (500000 - totalRead < 8 * 1024) {
            tryToRead = 500000 - totalRead;
        }
        return tryToRead;
    }

    public static void main(String[] args) throws IOException {
   
    	HttpClient httpclient = new DefaultHttpClient();
    	int connection_timeout=8*1000;
    	int waitResponse_timeout=10*1000;
		//设置一些链接属性
		//链接超时
    	httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connection_timeout);
		//请求超时
    	httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, waitResponse_timeout);
       
		//代理
    	//ConnRoutePNames.DEFAULT_PROXY
    	String hostName="192.168.88.1";
    	HttpHost proxyServer=new HttpHost(hostName, 3128);
    	httpclient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyServer);
		URL url=new URL("http://hc.apache.org/httpcomponents-client-ga/tutorial/html/connmgmt.html");
		//httpclient.getParams().setParameter(arg0, arg1);
		HttpGet httpget = null;
		InputStream inputStream=null;
		try {
			httpget = new HttpGet(url.toURI());
			
		} catch (URISyntaxException e) {
		}
		HttpResponse response = httpclient.execute(httpget);
		HttpEntity entity = response.getEntity();
		if (entity != null) {
		      inputStream = entity.getContent();
		  
		}
		System.out.print(CommonUtil.getStrFromInputStream(inputStream, null));

    }
}
